#!/bin/bash

#                                              #
# get performance statistics from local server #							
# (c) 2010 2011 At Consulting                  #
# mmartynov@at-consulting.ru                   #
#                                              #

SCRIPT_VERSION=2.0.0-rhel
KEYS=`echo $@ | sed 's/start//;s/stop//;s/status//'`
DEVICES=
INTERVAL=2
ACTION=
BINDIR=`dirname $0`/bin
EXEC_SCRIPT=$BINDIR/check_performance_exe.sh
CONFIGURE_SCRIPT=$BINDIR/check_configure_exe.sh
export BINDIR
export LOGDIR=`dirname $0`/log
export OUTDIR=`dirname $0`/out
export LOGFILE=$LOGDIR/check_performance.log
export ERRFILE=$LOGDIR/check_performance.err
export PIDFILE=$LOGDIR/check_performance.pid
mkdir -p $LOGDIR $OUTDIR $BINDIR

MAIL_ADDR=

usage()
{
cat <<EOF
Usage: `basename $0` [OPTION] start|stop|status
Known values for OPTION are:
     -h, --help
     -v, --version
  start     
     -a, --all-utilites
     --vmstat  (default)
     --netstat (default)
     --free
     --mpstat
     --iostat
     --sar	(not supported)
     --time-interval=TIME (default: TIME=2)
  stop 
     --send-mail[=MAIL] (default: MAIL=mmartynov@at-consulting.ru)
EOF
    exit $1
}

check_utilites() {
    while [ $# -gt 0 ]
    do
	[ ! -f "`which $1`" ] && { echo "ERR: not find \"$1\" utilities. exit 1"; exit 1; }
	shift
    done
}

#########################################################################################
#########################################################################################

create_configure_script() {
cat <<'CREATE_EOF'
#!/bin/bash

PATH=/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin:/usr/sfw/bin:/usr/ucb:/usr/bin/X11
export PATH

LANG=C
export LANG
>$OUTDIR/configure.out

real_host=`hostname`
os_version=`uname -sr`
total_mem=`free | grep "^Mem:" | awk '{print $2}'`
total_swap=`free | grep "^Swap:" | awk '{print $2}'`
{
echo -e "HOST:\t$real_host"
echo -e "OS:\t$os_version"
echo -e "Mem:\t$total_mem\t\tSwap:\t$total_swap"
echo -e "\nCPU Info:\n----------------------\n"
	dmesg | egrep "^CPU[0-9]:|^Memory:"
	cat /proc/cpuinfo
} | tee -a $OUTDIR/configure.out

exit 0
CREATE_EOF
}

create_my_netstat() {
cat <<'CREATE_EOF'
#!/bin/bash

interval=$1
[ -z "$interval" ] && interval=1

while :
do
awk '/eth0/ { split($1,a,":"); print a[2], $9}' /proc/net/dev
sleep $interval
done
CREATE_EOF
}

create_exec_script() {
cat <<'CREATE_EOF'
#!/bin/bash

PATH=/usr/local/bin:/usr/bin:/bin:/sbin:/usr/sbin:/usr/sfw/bin:/usr/ucb:/usr/bin/X11
export PATH

LANG=C
export LANG

echo "options: \"$@\""
INTERVAL=$1
if [ -z "$INTERVAL" ]; then
    echo "Alarm! not define TIME INTERVAL"
    exit 1
fi
if [ -z "$OUTDIR" -o -z "PIDFILE" ]; then
    echo "Alarm! not define OUTDIR or PIDFILE"
    exit 1
fi

VMSTAT=
FREE=
MPSTAT=
IOSTST=
SAR=
NETSTAT=

echo $@ | grep VMSTAT >/dev/null
[ $? -eq 0 ] && VMSTAT="ok"	

echo $@ | grep FREE >/dev/null
[ $? -eq 0 ] && FREE="ok"

echo $@ | grep MPSTAT >/dev/null
[ $? -eq 0 ] && MPSTAT="ok"

echo $@ | grep IOSTAT >/dev/null
[ $? -eq 0 ] && IOSTAT="ok"

echo $@ | grep SAR >/dev/null
[ $? -eq 0 ] && SAR="ok"

echo $@ | grep NETSTAT >/dev/null
#[ $? -eq 0 ] && NETSTAT="ok"

HEADER="o o o =================== o o o"

[ ! -z "$VMSTAT" ] && echo "$HEADER" > $OUTDIR/vmstat.out
[ ! -z "$FREE" ] && echo "$HEADER" > $OUTDIR/free.out
[ ! -z "$MPSTAT" ] && echo "$HEADER" > $OUTDIR/mpstat.out
[ ! -z "$IOSTAT" ] && echo "$HEADER" > $OUTDIR/iostat.out
[ ! -z "$SAR" ] && echo "$HEADER" > $OUTDIR/sar.out
[ ! -z "$NETSTAT" ] && echo "$HEADER" > $OUTDIR/netstat.out


###################################################################

#### aggregate functions
function max () {
    awk 'BEGIN{s=0}{if ($'$1' && s<$'$1') s=$'$1'}END{print s}'
}
function min () {
    awk 'BEGIN{s=999999}{if ($'$1' && s>$'$1') s=$'$1'} END{if (s==999999) s=0; print s}'
}
function avg () {
    awk 'BEGIN{s=0}{if ($'$1') s=s+$'$1'}END{print s/NR}'
}
function sum () {
    awk 'BEGIN{s=0}{if ($'$1') s=s+$'$1'}END{print s}'
}

function update_pidfile() {
    cat $PIDFILE | sed s/^ITERATION=.*$/ITERATION=$ITERATION/ | sed s/^TOTAL_TIME=.*$/TOTAL_TIME=$SECONDS/ > $PIDFILE.new
    mv $PIDFILE.new $PIDFILE
}

function cecho() {
while read line
do
    echo "`date '+%H:%M:%S'` $line"
done
}

ITERATION=0
while :
do
    ITERATION=`expr $ITERATION + 1`
    TIME=`date '+%Y.%m.%d %H:%M'`
    echo "iteration $ITERATION"
    echo -e "\tbegin `date +%H:%M:%S`"
    
    pvmstat=	
    if [ ! -z "$VMSTAT" ]; then
 	vmstat $INTERVAL 11 | cecho >> $OUTDIR/vmstat.out &
	pvmstat=$!
    fi
    if [ ! -z "$FREE" ]; then
 	free -s $INTERVAL | cecho >> $OUTDIR/free.out &
	pfree=$!
    fi
    if [ ! -z "$MPSTAT" ]; then
	mpstat -P ALL $INTERVAL 15 | cecho >> $OUTDIR/mpstat.out &
	pmpstat=$!
    fi
    if [ ! -z "$IOSTAT" ]; then
 	iostat -x $INTERVAL 15 | cecho >> $OUTDIR/iostat.out &
	piostat=$!
    fi
    if [ ! -z "$NETSTAT" ]; then
	#netstat -I $INTERVAL | cecho >> $OUTDIR/netstat.out &
	$BINDIR/my_netstat.sh $INTERVAL | cecho >> $OUTDIR/netstat.out &
	pnetstat=$!
    fi

    [ ! -z "$SAR" ] && echo "." | cecho >> $OUTDIR/sar.out

	if [ ! -z "$pvmstat" ]; then
		wait $pvmstat
	else	
		sleep `expr $INTERVAL \* 10`
	fi
	kill $pfree $piostat $pmpstat $pnetstat
    update_pidfile

done

exit 0
CREATE_EOF
}

#########################################################################################
#########################################################################################

while [ $# -gt 0 ]
do
	case "$1" in
	--*=*)  optname=`echo "$1" | sed 's/=.*$//'`
		optarg=`echo "$1" | sed 's/^.*=//'`
		;;
	*)  optname=$1
	    optarg=
	    ;;
	esac

	case "$optname" in
	-h|--help)
			usage 0
			;;
	-v|--version)
			echo $SCRIPT_VERSION
			;;
	-a|--all-utilites)
			check_utilites vmstat free mpstat iostat sar netstat
			DEVICES=" VMSTAT FREE MPSTAT IOSTAT SAR NETSTAT"
			;;
	--vmstat)	check_utilites vmstat
			DEVICES="$DEVICES VMSTAT"
			;;
	--free)		check_utilites free
			DEVICES="$DEVICES FREE"
			;;
	--mpstat)	check_utilites mpstat
			DEVICES="$DEVICES MPSTAT"
			;;
	--iostat)	check_utilites iostat
			DEVICES="$DEVICES IOSTAT"
			;;
	--sar)		check_utilites sar
			DEVICES="$DEVICES SAR"
			;;
	--netstat)	check_utilites netstat
			DEVICES="$DEVICES NETSTAT"
			;;
	--time-interval)
			[ -z "$optarg" ] && { echo "ERR: unknown TIME interval. exit 1"; exit 1; }
			INTERVAL=$optarg
			;;
	--send-mail)
			#[ -z "$optarg" ] && optarg='mmartynov@vrn.beeline.ru'
			[ -z "$optarg" ] && optarg='mmartynov@at-consulting.ru'
			MAIL_ADDR=$optarg
			;;
	start|stop|status)
			ACTION=$1
			;;
	*)
		echo "unknown option $1"
		usage 1
		;;
	esac
	shift
done

# 1-running, 0-not running
function status() {
    pcount=`pgrep -f "$EXEC_SCRIPT" | wc -l`
    if [ "$pcount" -gt 0 ]; then
	return 1
    else
	return 0
    fi
}

[ -z "$ACTION" -o -z "$INTERVAL" ] && usage 1
[ -z "`echo $INTERVAL | sed -n /^[0-9][0-9]*$/p`" ] && { echo time-interval is not number; exit 1; }
[ -z "$DEVICES" ] && DEVICES=" VMSTAT NETSTAT"

if [ "$ACTION" == "status" ]; then
	status
	if [ $? -ne 0 ]; then
		echo "programm is running."
		#if [ "`cat $PIDFILE | sed -n s/^PID=//p`" -eq "`pgrep -f "$EXEC_SCRIPT"`" ]; then
		echo "-----------------------"
		cat $PIDFILE
		echo "-----------------------"
		#fi
	else
		echo "progrmm is not running."
	fi
	exit 0
fi

if [ "$ACTION" == "start" ]; then
	status
	if [ $? -ne 0 ]; then
	    echo "programm already running."
	    exit 1
	fi
	
	rm -f $OUTDIR/*.out	
	rm -f $LOGDIR/*.log $LOGDIR/*.err
	create_exec_script > $EXEC_SCRIPT
	create_configure_script > $CONFIGURE_SCRIPT
	create_my_netstat > $BINDIR/my_netstat.sh
	chmod +x $EXEC_SCRIPT $CONFIGURE_SCRIPT $BINDIR/my_netstat.sh

	echo -e "START: `date`\n================================" >> $LOGFILE
	> $PIDFILE
	> $ERRFILE
	echo "configuring ..." | tee -a $LOGFILE
	"$CONFIGURE_SCRIPT" $DEVICES >>$LOGFILE 2>>$ERRFILE
	[ $? -ne 0 ] && { echo "error! view $LOGFILE"; exit 1; }
	echo "starting ..." | tee -a $LOGFILE; sleep 1 
	"$EXEC_SCRIPT" $INTERVAL $DEVICES >>$LOGFILE 2>>$ERRFILE &
	mypid=$!
	{
	echo "PID=$mypid"
	echo "START_DATE=`date`"
	echo "LOGNAME=$LOGNAME"
	echo "DEVICES=$DEVICES"
	echo "ITERATION=0"
	echo "TOTAL_TIME=0"
	} >> $PIDFILE
fi

if [ "$ACTION" == "stop" ]; then
	status
	if [ $? -eq 0 ]; then
	    echo "programm already stopped."
	    exit 1
	fi
	echo "stopped ..." | tee -a $LOGFILE; sleep 1
	pkill -f `basename $EXEC_SCRIPT`
	
	echo "correct out-files ..." | tee -a $LOGFILE
	id=0
	false
	while [ $? != 0 ]
	do
        echo -n "Enter server id [0-9]: "
        read id
        echo $id | grep -s "^[0-9]$" > /dev/null
	done

	cd $OUTDIR
	if [ -f vmstat.out ]; then
		sed -n '3p' vmstat.out  | sed '1s/ \([a-z]\)/ '$id'\.\1/g' > vmstat.new.out
		cat vmstat.out | egrep -v "=====|-----" | sed '/r  b   swpd/{n;d;}' | sed '/r  b   swpd/d' >> vmstat.new.out
		mv vmstat.new.out vmstat.out
	fi	
	if [ -f free.out ]; then
		sed -n 2p free.out | sed '1s/ \([a-z]\)/ '$id'\.\1/g' > free.mem.out
		grep "Mem:" free.out | sed s/Mem:// >> free.mem.out
		sed -n 2p free.out | awk '{print $1, $2, $3, $4}' | sed '1s/ \([a-z]\)/ '$id'\.\1/g' > free.swap.out
		grep "Swap:" free.out | sed s/Swap:// >> free.swap.out
		rm free.out
	fi
	if [ -f netstat.out ]; then
        echo "- RX TX" | sed '1s/ \([a-z]\)/ '$id'\.\1/g' > netstat.new.out
        cat netstat.out | awk '{d2=$2-s2; d3=$3-s3; s2=$2; s3=$3; print $1, d2, d3}' | sed 1,2d >> netstat.new.out
		mv netstat.new.out netstat.out
	fi	
	cd - > /dev/null

	RESFILE=$OUTDIR/`basename $0 .sh`.`hostname`.`date '+%y%m%d%H%M'`.tar
	tar -cf $RESFILE $OUTDIR/*.out $LOGDIR/*
	if [ ! -z "$MAIL_ADDR" ]; then
		echo "send mail: $MAIL_ADDR" >> $LOGFILE
		echo "see attached file" | mutt -s "check_performance: `hostname`" -a $RESFILE $MAIL_ADDR
	fi
	echo -e "STOP: `date`\n\n" >> $LOGFILE
	rm $BINDIR/*.sh	
	#rm $OUTDIR/*.out
fi
